#!/bin/bash
#SBATCH -o ./logs/slurm-%j.out

set -e

module add cuda/10.2

# cuDNN settings.
export LD_LIBRARY_PATH=/THL5/home/kangbo1/haofu/pytorch/libs/cudnn/lib64:$LD_LIBRARY_PATH
export LD_RUN_PATH=/THL5/home/kangbo1/haofu/pytorch/libs/cudnn/lib64:$LD_RUN_PATH
# NCCL settings.
export LD_LIBRARY_PATH=/THL5/home/kangbo1/haofu/pytorch/libs/nccl/lib:$LD_LIBRARY_PATH
export LD_RUN_PATH=/THL5/home/kangbo1/haofu/pytorch/libs/nccl/lib:$LD_RUN_PATH
export PKG_CONFIG_PATH=/THL5/home/kangbo1/haofu/pytorch/libs/nccl/lib/pkgconfig:$PKG_CONFIG_PATH

source activate /THL5/home/kangbo1/haofu/venvs/pytorch

job_id=$SLURM_JOBID

yhrun -N $1 -n $1 -p TH_GPU /THL5/home/kangbo1/haofu/venvs/pytorch/bin/python ./main.py --evaluate --multiprocessing-distributed /THL5/home/kangbo1/haofu/DATASETS/ImageNet2012

if [ ! -d ./dist_conf ]; then
    mkdir -p ./dist_conf
fi
mv dist_url_${job_id}.txt ./dist_conf/
